import os
import time
import json
import re


blacklist_languages = set([None, "Markdown", "HTML"])


blacklist_topics = set([
  "development",
  "programming",
  "computer-science",
  "education",
  "learn",
  "course",
  "interview",
  "interview-questions",
  "awesome",
  "awesome-list",
  "best-practices",
  "algorithm",
  "leetcode",
  "chinese",
])


blacklist_owners = set([
  "doocs",
  "fighting41love",
  "bannedbook",
  "GokuMohandas",
  "Alvin9999"
])


blacklist_name_regex = re.compile(r"(handbook|cookbook|storybook|roadmap|tutorial|example|question|interview|learning|beginner|academic|days|things|chinese)", re.IGNORECASE)


blacklist_project = set([
  "build-web-application-with-golang", #349
  "awesome-cheatsheets", #392
])


def check_valid(item):
  [owner, name] = item["full_name"].split("/")
  return item["language"] not in blacklist_languages \
    and len(set(item["topics"]).intersection(blacklist_topics)) == 0 \
    and owner not in blacklist_owners \
    and len(blacklist_name_regex.findall(name)) == 0 \
    and name not in blacklist_project \


def filter(src_file, resource_file, popular_file):
  try:
    items = []

    with open(src_file, 'r') as f:
      items = json.load(f)
    
    populars = []
    resources = []

    for (index, item) in enumerate(items):
      report_item = {
        "sort_index": index,
        "full_name": item["full_name"],
        "description": item["description"],
        "html_url": item["html_url"],
        "language": item["language"],
        "tags": ",".join(item["topics"]),
        "stargazers_count": item["stargazers_count"],
        "forks_count": item["forks_count"],
        "watchers_count": item["watchers_count"],
        "created_at": item["created_at"],
        "pushed_at": item["pushed_at"],
        "homepage": item["homepage"],
        "license": item["license"]["spdx_id"] if item["license"] else "",
        "size": item["size"],
      }
      if check_valid(item):
        populars.append(report_item)
      else:
        resources.append(report_item)

    print(f"Filtered items: {len(populars)}/{len(items)}")

    os.makedirs(os.path.dirname(popular_file), exist_ok=True)
    with open(popular_file, 'w') as f:
      json.dump(populars, f, indent=2)

    os.makedirs(os.path.dirname(resource_file), exist_ok=True)
    with open(resource_file, 'w') as f:
      json.dump(resources, f, indent=2)

  except Exception as e:
    print(e)


if __name__ == "__main__":
  for src_file in os.listdir('data/most_stars'):
    if re.match(r"^\d{4}_\d{2}_\d{2}\.json$", src_file):
      filter(f'data/most_stars/{src_file}', f'data/resources/{src_file}', f'data/populars/{src_file}')
  